/**
 * Copyright 2011 FeedDreamwork SIG
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package org.feeddreamwork.fetcher;

import java.io.*;
import java.net.*;
import java.util.*;

import org.feeddreamwork.*;

public class HttpFetcher extends HttpFetcherBase {
	public HttpFetcher(String url) throws MalformedURLException {
		super(url);
	}

	@Override
	public String getContent() throws IOException {
		BufferedReader reader = null;
		int retry = ApplicationProperty.HTTP_RETRY_TIMES;
		int fetchLimit = ApplicationProperty.HTTP_FETCH_LIMIT;

		while (true)
			try {
				reader = new BufferedReader(new InputStreamReader(url.openStream(), getEncoding()));
				String line;
				StringBuilder contentBuilder = new StringBuilder();
				while ((line = reader.readLine()) != null && contentBuilder.length() <= fetchLimit)
					contentBuilder.append(line).append('\n');
				String content = contentBuilder.toString();

				if (content.length() > fetchLimit)
					content.substring(0, fetchLimit);
				if (content.contains("400 Bad Request") && content.contains("nginx"))
					throw new IOException();
				return content;
			} catch (IOException e) {
				retry--;
				if (retry == 0)
					throw e;
			} finally {
				if (reader != null)
					reader.close();
			}
	}

	private String getEncoding() throws IOException {
		BufferedReader reader = null;
		String encoding = null;

		try {
			URLConnection connection = url.openConnection();
			Map<String, List<String>> header = connection.getHeaderFields();
			for (Map.Entry<String, List<String>> entry : header.entrySet())
				if (entry.getKey().toLowerCase().equals("content-type")) {
					String item = entry.getValue().toString().toLowerCase();
					if (item.contains("charset")) {
						encoding = extractEncoding(item);
						if (!Utils.isNullOrEmpty(encoding))
							return encoding;
					}
				}

			reader = new BufferedReader(new InputStreamReader(
					connection.getInputStream()));
			String line;
			while ((line = reader.readLine()) != null) {
				line = line.toLowerCase();
				if (line.contains("charset") || line.contains("encoding")) {
					encoding = extractEncoding(line);
					if (!Utils.isNullOrEmpty(encoding))
						return encoding;
				}
			}

			return ApplicationProperty.DEFAULT_ENCODING;
		} finally {
			if (reader != null)
				reader.close();
		}
	}

	private String extractEncoding(String source) {
		if (source.contains("gb2312"))
			return "GBK";
		if (source.contains("gbk"))
			return "GBK";
		if (source.contains("big5") || source.contains("big-5"))
			return "Big5";
		if (source.contains("utf-8") || source.contains("utf8"))
			return "UTF-8";
		return null;
	}
}
